import importlib
import inspect
import subprocess
import time
from utils.redis_helper import RedisHelper
from utils.utilities import message_head
import utils.general_settings as general_settings
import psutil


def check_process_alive(alive_pid_dict: dict):
    """根据pid和创建时间检测进程是否存在
    return：存在的进程号的list
    """
    current_pid_list = psutil.pids()
    exist_process_dict = {}
    for key in alive_pid_dict.keys():
        exist_process_dict[key] = []
        for (pid, create_time) in alive_pid_dict[key]:
            # 如果pid在系统活动进程的列表中
            if pid in current_pid_list:
                # 如果此pid对应的进程创建时间与传入的参数相同，则此进程为本程序创建且存活，保存信息
                if psutil.Process(pid).create_time() == create_time:
                    exist_process_dict[key].append((pid, psutil.Process(pid).create_time()))
            else:
                continue
    return exist_process_dict


if __name__ == "__main__":
    print(message_head(0) + "主程序启动")
    # 创建redis实例，获取任务队列信息
    redis = RedisHelper(host=general_settings.redis_host, port=general_settings.redis_port)
    # 存活进程的词典，key：脚本文件名 value：list类型，元素为tuple（进程号，创建时间）
    alive_pid_dict = {}

    sleep_count = 30
    while True:
        general_settings = importlib.reload(general_settings)
        urgent_queue_dict = {}
        normal_queue_dict = {}
        max_thread_dict = {}
        # 获取全部的配置类，并提取其中的队列名组成列表供检查队列是否有任务
        configs = inspect.getmembers(general_settings, inspect.isclass)
        for (name, _) in configs:
            config = getattr(general_settings, name)
            urgent_queue_dict[config().urgent_task_queue] = config().worker_file
            normal_queue_dict[config().task_queue] = config().worker_file
            if config().worker_file not in alive_pid_dict.keys():
                alive_pid_dict[config().worker_file] = []
            max_thread_dict[config().worker_file] = config().max_thread

        subprocess_count = 0
        alive_pid_dict = check_process_alive(alive_pid_dict)
        for key in alive_pid_dict.keys():
            for (pid, create_time) in alive_pid_dict[key]:
                subprocess_count += 1

        for queue_dict in [urgent_queue_dict, normal_queue_dict]:
            for queue in queue_dict.keys():
                # 队列还有任务时
                while redis.get_list_length(queue) > 0:
                    current_count = len(alive_pid_dict[queue_dict[queue]])
                    # 检查队列对应的脚本文件的存活进程数量是否大于等于该爬虫文件所允许运行数量的最大值
                    # 大于等于则跳出，执行下一个队列的任务创建，小于则对本队列创建新任务并记录进程信息
                    if current_count >= max_thread_dict[queue_dict[queue]]:
                        break
                    else:
                        # 系统空闲内存大于设置的最小值，继续创建进程，否则等待
                        while True:
                            if float(psutil.virtual_memory().free / 1024 / 1024) > general_settings.worker_lowest_memory_MB:
                                run_proc = subprocess.Popen("python " + queue_dict[queue])
                                process = psutil.Process(run_proc.pid)
                                alive_pid_dict[queue_dict[queue]].append((process.pid, process.create_time()))
                                time.sleep(5)
                                subprocess_count += 1
                                break
                            else:
                                print(message_head(0) + "系统可用内存低于设置的最小值.....")
                                time.sleep(30)
        time.sleep(10)

        if sleep_count >= 30:
            print('urgent_queue_dict:', urgent_queue_dict)
            print('normal_queue_dict:', normal_queue_dict)
            print('alive_pid_dict:', alive_pid_dict)
            print('max_thread_dict:', max_thread_dict)
            if subprocess_count == 0:
                print(message_head(0) + "主程序等待任务......")
            sleep_count = 0
        else:
            sleep_count += 1
